{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fbb53c1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#将vecs格式的数据集转换为bin格式的\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "import struct\n",
    "\n",
    "#向量的类型\n",
    "type = np.float32 # np.uint8\n",
    "dataset_root_path = \"/home/ljl/Code/dataset/vector-ssd/deep\"\n",
    "vecs_path = dataset_root_path+\"/1M.fvecs\"\n",
    "bin_path = dataset_root_path+\"/1M.fbin\"\n",
    "\n",
    "override = True #如果bin_path已经存在，是否覆盖写"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42546472",
   "metadata": {},
   "outputs": [],
   "source": [
    "#判断vecs_path是否存在，以及bin_path是否已存在\n",
    "if not os.path.exists(vecs_path):\n",
    "    print(\"vecs_path:\",vecs_path,\"不存在\")\n",
    "    quit()\n",
    "if not override and os.path.exists(bin_path):\n",
    "    print(\"bin_path:\",bin_path,\"已存在\")\n",
    "    quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3c6153ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vecs_file_size: 388000000\n",
      "vector_count: 1000000\n",
      "dim: 96\n",
      "feature_size: 4\n",
      "vector_size: 384\n"
     ]
    }
   ],
   "source": [
    "#读取vecs_path文件的大小.我们这里假设每个向量前面的维度数都是一样的\n",
    "vecs_file_size = os.path.getsize(vecs_path)\n",
    "vector_count = 0\n",
    "dim = 0\n",
    "feature_size = int(np.dtype(type).itemsize)\n",
    "vector_size = 0\n",
    "\n",
    "with open(vecs_path, 'rb') as f:\n",
    "    line = f.read(4)\n",
    "    dim = struct.unpack('i', line)[0]\n",
    "    vector_size = feature_size * dim\n",
    "    #如果vecs_file_size不能整除vector_size，则说明vecs_file_size有误\n",
    "    assert vecs_file_size % (vector_size+4) == 0\n",
    "    vector_count = int(vecs_file_size / (vector_size+4))\n",
    "\n",
    "print(\"vecs_file_size:\", vecs_file_size)\n",
    "print(\"vector_count:\", vector_count)\n",
    "print(\"dim:\", dim)\n",
    "print(\"feature_size:\", feature_size)\n",
    "print(\"vector_size:\", vector_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a3c58c73",
   "metadata": {},
   "outputs": [],
   "source": [
    "#开始转换，先写入4字节的vector_count和dim\n",
    "vecs_f = open(vecs_path, \"rb\")\n",
    "bin_f = open(bin_path, \"wb\")\n",
    "\n",
    "bin_f.write(struct.pack(\"I\", vector_count))\n",
    "bin_f.write(struct.pack(\"I\", dim))\n",
    "\n",
    "for i in range(vector_count):\n",
    "    vecs_f.seek(4, 1)\n",
    "    line = vecs_f.read(vector_size)\n",
    "    bin_f.write(line)\n",
    "\n",
    "vecs_f.close()\n",
    "bin_f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e3ce81b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
